package com.strat.searcher.analisys.tokenize;

import java.util.LinkedList;
import java.util.List;

public class TokenUtil {

    public static List<Token> filtredTokens(List<Token> tokens, TokenType tp) {
        List<Token> outTokens = new LinkedList<Token>();
        for (Token token : tokens) {
            if (token.getType().equals(tp)) {
                outTokens.add(token);
            }
        }
        return outTokens;
    }

    public static List<Token> tokenizeText(TextTokenizer tokenizer) {
        List<Token> tokens = new LinkedList<Token>();
        Token token = null;
        while ((token = tokenizer.nextToken()) != null) {
            tokens.add(token);
        }
        return tokens;
    }
}
